#looking_for_status_messages_(input year).py
#method to look for new status messages in json data files obtained from the USPTO
#written by Nick Shine
#November 2020

import json
import sys
import csv
import pandas as pd
import gc

#this number changes depending on the year, which contains a differing amount of data
numberOfYearsOfJSONdata = 42
#starting .json file
pedsJSONfile = '1900.json'

#variable for the number of apps count check
numberOfAppsTotal = 0

#variables to count apps with a missing app status message/art units
missingAppStatusCount = 0
missingGroupArtUnitCount = 0

#variable for the month counts
janCount = 0
febCount = 0
marCount = 0
aprCount = 0
mayCount = 0
junCount = 0
julCount = 0
augCount = 0
sepCount = 0
octCount = 0
novCount = 0
decCount = 0	

#throw away variable
dummy = 0

#variables to hold counts for each type of message
patentedCasesCount = 0
abandonedCasesCount = 0
docketedNewCaseCount = 0
appDispatchedCaseCount = 0
appPreexamCaseCount = 0
prosSuspendedCaseCount = 0
expAbandonedCasesCount = 0
nonFinalMailedCasesCount = 0
responseNonFinalCasesCount = 0
abandonedIncompleteCasesCount = 0
abandonedFailureToPayCasesCount = 0
abandonedDrawingsCasesCount = 0
awaitingResponseCasesCount = 0
noticeAppealFiledCasesCount = 0
interferenceCasesCount = 0
finalRejectionMailedCasesCount = 0
expAbandondedDuringCasesCount = 0
interferenceBoardCasesCount = 0
advisoryActionCasesCount = 0
onAppealAwaitingCasesCount = 0
abandondedExaminerAnswerCasesCount = 0
exParteCasesCount = 0
publicationFeePayVerifiedCasesCount = 0
withdrawnAbandonmentCasesCount = 0
appInCourtCaseCasesCount = 0
appealBriefEnteredCasesCount = 0
briefAnswerMailedCasesCount = 0
expired1362CasesCount = 0
appealBoardDecisionCasesCount = 0
tcReturnOfAppealCasesCount = 0
courtProTerminatedCasesCount = 0
requestReconsidCasesCount = 0
appReturnedBackCasesCount = 0
missassignedAppNumCasesCount = 0
clarificationCasesCount = 0
noaAppRecdInOfficeOfPubCasesCount = 0
responseAfterFinalFwdToExamCasesCount = 0
pubFeePayRcvedCasesCount = 0
respExParteQuayleEntFwdCasesCount = 0
amendArgAftrBofAppDecCasesCount = 0
bOfAppDecRendAftReqReconCasesCount = 0
amendAfterNotOfAppealCasesCount = 0
awaitTCrespIssueFeeNotPaidCasesCount = 0
awaitTCrespIssueFeeRecvdCasesCount = 0
awaitTCrespIssueFeeVerifiedCasesCount = 0
examAnsrReplyBriefRemandMailedCasesCount = 0
appealDismsdWithdrwnCasesCount = 0
appealRdyForRevwCasesCount = 0
withdrwFromIssieAwtActnCasesCount = 0
repBrfOrSupFwdToExamrCasesCount = 0
interfDecsnPriortyRendrdByBoardCasesCount = 0
interfDispchToExamrCasesCount = 0
finalRejCountdNotMailedCasesCount = 0
preIntrvwCommMailedCasesCount = 0
speacialNewCasesCount = 0
remandToExmnrFromBofAppCasesCount = 0
allowCountedCasesCount = 0
repBrfFiledFwdToBPAICasesCount = 0
nonFinalRejCountdNotMailedCasesCount = 0
undocketedTrack1CasesCount = 0
abandondedRestoredCasesCount = 0
nfoaActionEntereadyExmrCasesCount = 0
oldCaseAddedTrackingCasesCount = 0
seReadyForPubsProcessIFWCasesCount = 0

#loop to read each pedsJSONfile for the year and determine if and what the status messages are
for x in range(numberOfYearsOfJSONdata):

	with open(pedsJSONfile, 'r', encoding='utf-8') as json_file:
	  	data = json.load(json_file)

	totalNumberOfAppsCount = 0			

	for i in range(len(data['PatentData'])):
		
		if 'groupArtUnitNumber' in data['PatentData'][i]['patentCaseMetadata']:
			if data['PatentData'][i]['patentCaseMetadata']['groupArtUnitNumber']['value'][0:2] == '16':
				dummy += 1
		else:
			#print('***********************************************************')
			#print('')
			#print(i,'Group Art Unit is NULL', 'App Id is', (data['PatentData'][i]['patentCaseMetadata']['applicationNumberText']['value']), 'json file is', pedsJSONfile )
			#print('')
			#print('***********************************************************')
			missingGroupArtUnitCount = (missingGroupArtUnitCount + 1)

		if 'applicationStatusCategory' in data['PatentData'][i]['patentCaseMetadata']:
			if data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '01':
				janCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '02':
				febCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '03':
				marCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '04':
				aprCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '05':
				mayCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '06':
				junCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '07':
				julCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '08':
				augCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '09':
				sepCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '10':
				octCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '11':
				novCount += 1
			elif data['PatentData'][i]['patentCaseMetadata']['applicationStatusDate'][5:7] == '12':
				decCount += 1
		else:
			print('***********************************************************')
			print('')
			print(i,'App Status is NULL')
			print('')
			print('***********************************************************')
			missingAppStatusCount = (missingAppStatusCount + 1)
		
		status = data['PatentData'][i]['patentCaseMetadata']['applicationStatusCategory']

		if status == ('Patented Case'): 
			patentedCasesCount += 1
		elif status == ('Application Dispatched from Preexam, Not Yet Docketed'):
			appDispatchedCaseCount += 1
		elif status == ('Docketed New Case - Ready for Examination'):
			docketedNewCaseCount += 1
		elif status == ('Abandoned  --  Failure to Respond to an Office Action'):
			abandonedCasesCount += 1
		elif status == ('Application Undergoing Preexam Processing'):
			appPreexamCaseCount += 1
		elif status == ('Prosecution Suspended'):
			prosSuspendedCaseCount += 1
		elif status == ('Expressly Abandoned  --  During Examination'):
			expAbandonedCasesCount += 1
		elif status == ('Non Final Action Mailed'):
			nonFinalMailedCasesCount += 1
		elif status == ('Response to Non-Final Office Action Entered and Forwarded to Examiner'):
			responseNonFinalCasesCount += 1
		elif status == ('Abandoned  --  Incomplete Application (Pre-examination)'):
			abandonedIncompleteCasesCount += 1
		elif status == ('Abandoned  --  Failure to Pay Issue Fee'):
			abandonedFailureToPayCasesCount += 1
		elif status == ('Abandonment for Failure to Correct Drawings/Oath/NonPub Request'):
			abandonedDrawingsCasesCount += 1
		elif status == ('AWAITING RESPONSE FOR INFORMALITY, FEE DEFICIENCY OR CRF ACTION'):
			awaitingResponseCasesCount += 1
		elif status == ('Notice of Appeal Filed'):
			noticeAppealFiledCasesCount += 1
		elif status == ('Interference -- Initial Memorandum'):
			interferenceCasesCount += 1
		elif status == ('Final Rejection Mailed'):
			finalRejectionMailedCasesCount += 1
		elif status == ('Expressly Abandoned  --  During Publication Process'):
			expAbandondedDuringCasesCount += 1
		elif status == ('Interference -- Declared by Board of Interferences'):
			interferenceBoardCasesCount += 1
		elif status == ('Advisory Action Mailed'):
			advisoryActionCasesCount += 1
		elif status == ('On Appeal -- Awaiting Decision by the Board of Appeals'):
			onAppealAwaitingCasesCount += 1
		elif status == ('Abandoned  --  After Examiner\'s Answer or Board of Appeals Decision'):
			abandondedExaminerAnswerCasesCount += 1
		elif status == ('Ex parte Quayle Action Mailed'):
			exParteCasesCount += 1
		elif status == ('Publications -- Issue Fee Payment Verified'):
			publicationFeePayVerifiedCasesCount += 1
		elif status == ('Withdrawn Abandonment, awaiting examiner action'):
			withdrawnAbandonmentCasesCount += 1
		elif status == ('Application Involved in Court Proceedings'):
			appInCourtCaseCasesCount += 1
		elif status == ('Appeal Brief (or Supplemental Brief) Entered and Forwarded to Examiner'):
			appealBriefEnteredCasesCount += 1
		elif status == ('Examiner\'s Answer to Appeal Brief Mailed'):
			briefAnswerMailedCasesCount += 1
		elif status == ('Patent Expired Due to NonPayment of Maintenance Fees Under 37 CFR 1.362'):
			expired1362CasesCount += 1
		elif status == ('Board of Appeals Decision Rendered'):
			appealBoardDecisionCasesCount += 1
		elif status == ('TC Return of Appeal'):
			tcReturnOfAppealCasesCount += 1
		elif status == ('Court Proceedings Terminated'):
			courtProTerminatedCasesCount += 1
		elif status == ('Request Reconsideration after Board of Appeals Decision'):
			requestReconsidCasesCount += 1
		elif status == ('Application Returned back to Preexam'):
			appReturnedBackCasesCount += 1
		elif status == ('Missassigned Application Number'):
			missassignedAppNumCasesCount += 1
		elif status == ('Sent to Classification contractor'):
			clarificationCasesCount += 1
		elif status == ('Notice of Allowance Mailed -- Application Received in Office of Publications'):
			noaAppRecdInOfficeOfPubCasesCount += 1
		elif status == ('Response after Final Action Forwarded to Examiner'):
			responseAfterFinalFwdToExamCasesCount += 1
		elif status == ('Publications -- Issue Fee Payment Received'):
			pubFeePayRcvedCasesCount += 1
		elif status == ('Response to Ex parte Quayle Action Entered and Forwarded to Examiner'):
			respExParteQuayleEntFwdCasesCount += 1
		elif status == ('Amendment / Argument after Board of Appeals Decision'):
			amendArgAftrBofAppDecCasesCount += 1
		elif status == ('Board of Appeals Decision Rendered after Request for Reconsideration'):
			bOfAppDecRendAftReqReconCasesCount += 1
		elif status == ('Amendment after notice of appeal'):
			amendAfterNotOfAppealCasesCount += 1
		elif status == ('Awaiting TC Resp., Issue Fee Not Paid'):
			awaitTCrespIssueFeeNotPaidCasesCount += 1
		elif status == ('Awaiting TC Resp, Issue Fee Payment Received'):
			awaitTCrespIssueFeeRecvdCasesCount += 1
		elif status == ('Awaiting TC Resp, Issue Fee Payment Verified'):
			awaitTCrespIssueFeeVerifiedCasesCount += 1
		elif status == ('Examiner\'s Answer to Reply Brief or Response to Remand Mailed'):
			examAnsrReplyBriefRemandMailedCasesCount += 1
		elif status == ('Appeal Dismissed / Withdrawn'):
			appealDismsdWithdrwnCasesCount += 1
		elif status == ('Appeal Ready for Review'):
			appealRdyForRevwCasesCount += 1
		elif status == ('Withdraw from issue awaiting action'):
			withdrwFromIssieAwtActnCasesCount += 1
		elif status == ('Reply Brief (or Supplemental Reply Brief) Forwarded to Examiner'):
			repBrfOrSupFwdToExamrCasesCount += 1
		elif status == ('Interference -- Decision on Priority Rendered by Board of Interferences'):
			interfDecsnPriortyRendrdByBoardCasesCount += 1
		elif status == ('Interference-Dispatch to Examiner'):
			interfDispchToExamrCasesCount += 1
		elif status == ('Final Rejection Counted, Not Yet Mailed'):
			finalRejCountdNotMailedCasesCount += 1
		elif status == ('Non Final Action Counted, Not Yet Mailed'):
			nonFinalRejCountdNotMailedCasesCount += 1
		elif status == ('Pre-Interview Communication Mailed'):
			preIntrvwCommMailedCasesCount += 1
		elif status == ('Special New'):
			speacialNewCasesCount += 1
		elif status == ('Remand to Examiner from Board of Appeals'):
			remandToExmnrFromBofAppCasesCount += 1
		elif status == ('Allowance Counted'):
			allowCountedCasesCount += 1
		elif status == ('Reply Brief filed and forwarded to BPAI'):
			repBrfFiledFwdToBPAICasesCount += 1
		elif status == ('ABANDONED - RESTORED'):
			abandondedRestoredCasesCount += 1
		elif status == ('UNDOCKETED TRACK 1'):
			undocketedTrack1CasesCount += 1
		elif status == ('Non-Final Action Entered (or Ready for Examiner Action)'):
			nfoaActionEntereadyExmrCasesCount += 1
		elif status == ('Patented File - (Old Case Added for File Tracking Purposes)'):
			oldCaseAddedTrackingCasesCount += 1
		elif status == ('SE ready for Pubs Processing -- Certificate in IFW'):
			seReadyForPubsProcessIFWCasesCount += 1

		else:
			print('***********************************************************')
			print('')
			print(i,'New App Status ')
			print('')
			print(data['PatentData'][i]['patentCaseMetadata']['applicationStatusCategory'], '\n')
			print('***********************************************************')	
			print('\n')
    #clears the data variable holding the json data, which gets very large
	data.clear()
    #clears all unused variables in case Python hasn't done it yet, the memory requirements are very large
	gc.collect()

	totalNumberOfAppsCount = i
	
	print('\n')
	print('The total number of apps with missing status is ', missingAppStatusCount)
	#print('\n')
	print('The total number of apps with missing art unit is ', missingGroupArtUnitCount)
	#print('\n')
	print('The total number of apps for', pedsJSONfile[0:4], ' is', totalNumberOfAppsCount)
	

	numberOfAppsTotal += totalNumberOfAppsCount

	#this code changes the string to switch between the differend years of JSON data from the USPTO to output
	#individual year data Excel files
	#-------------------------------2016 Dataset------------------------------------------------------------
	# if pedsJSONfile == '1900.json' :
	# 	pedsJSONfile = '1973.json'

	# elif pedsJSONfile == '1973.json' :
	# 	pedsJSONfile = '1981.json'

	# elif pedsJSONfile == '1982.json' :
	# 	pedsJSONfile = '1984.json'

	# elif pedsJSONfile == '1985.json' :
	# 	pedsJSONfile = '1989.json'

	# elif pedsJSONfile == '1990.json' :
	# 	pedsJSONfile = '1992.json'
	#-------------------------------2016 Dataset------------------------------------------------------------
	#-------------------------------2017 Dataset------------------------------------------------------------
	# if pedsJSONfile == '1900.json' :
	# 	pedsJSONfile = '1986.json'

	# elif pedsJSONfile == '1986.json' :
	# 	pedsJSONfile = '1988.json'

	# elif pedsJSONfile == '1990.json' :
	# 	pedsJSONfile = '1992.json'
	#-------------------------------2017 Dataset------------------------------------------------------------
	#-------------------------------2018 Dataset------------------------------------------------------------
	if pedsJSONfile == '1900.json' :
		pedsJSONfile = '1980.json'
	#-------------------------------2018 Dataset------------------------------------------------------------
	#-------------------------------2019 Dataset------------------------------------------------------------
	# if pedsJSONfile == '1900.json' :
	# 	pedsJSONfile = '1976.json'

	# elif pedsJSONfile == '1976.json' :
	# 	pedsJSONfile = '1981.json'

	# elif pedsJSONfile == '1981.json' :
	# 	pedsJSONfile = '1984.json'

	# elif pedsJSONfile == '1984.json' :
	# 	pedsJSONfile = '1986.json'
	#-------------------------------2019 Dataset------------------------------------------------------------
	#-------------------------------2020 Dataset------------------------------------------------------------
	# if pedsJSONfile == '1900.json' :
	# 	pedsJSONfile = '1972.json'

	# elif pedsJSONfile == '1972.json' :
	# 	pedsJSONfile = '1976.json'

	# elif pedsJSONfile == '1977.json' :
	# 	pedsJSONfile = '1989.json'

	# elif pedsJSONfile == '1991.json' :
	# 	pedsJSONfile = '1993.json'
	#-------------------------------2020 Dataset------------------------------------------------------------

	else:
		changeTheYearDigits = pedsJSONfile[0:4]
	#these prints are for testing purposes
	#print(changeTheYearDigits)
		oneYearIncrement = (pd.Timestamp(changeTheYearDigits) + pd.DateOffset(years=1)).strftime('%Y')
	#these prints are for testing purposes
	#print(oneYearIncrement)
		pedsJSONfile = oneYearIncrement + '.json'
	
	print('\n')
	if pedsJSONfile[0:4] != '2021':
		print('Year', pedsJSONfile[0:4])
		print('\n')
print('\n')
print('The total for Jan is', janCount)
#print('\n')
print('The total for Feb is', febCount)
#print('\n')
print('The total for Mar is', marCount)
#print('\n')
print('The total for Apr is', aprCount)
#print('\n')
print('The total for May is', mayCount)
#print('\n')
print('The total for June is', junCount)
#print('\n')
print('The total for July is', julCount)
#print('\n')
print('The total for Aug is', augCount)
#print('\n')
print('The total for Sept is', sepCount)
#print('\n')
print('The total for Oct is', octCount)
#print('\n')
print('The total for Nov is', novCount)
#print('\n')
print('The total for Dec is', decCount)
print('\n')
print('The total overall processed is', numberOfAppsTotal)